#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
# @Time : 2018-04-16 10:31 
# @Author : Woolei
# @File : Step2_getDetailPage2bigFile.py 

import requests
import json
from json import JSONDecodeError
import os
import logging
import pymysql
import time
from TandfBook.ProjectConfig import *

logging.basicConfig(level=logging.INFO,  # 最低输出
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%d %b %Y %H:%M:%S')

# AUTH = 'idtoken eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpcCI6IjIyMi4xODAuMTk1LjE1NCIsImNvbnRlbnRfY2Vuc29yaW5nX2VuYWJsZWQiOmZhbHNlLCJzY29wZSI6WyJwcm9kdWN0X2FjY2VzcyJdLCJ0ZXJyaXRvcnlfdHlwZSI6ImNvdW50cnkiLCJ1c2VyIjp7Il9pZCI6IjVhMzBkMGU0NTc0Mjk3MWIwMDljOTIwZiIsInVzZXJuYW1lIjoiMjIyLjE4MC4xOTUuMTU0IiwiZW1haWwiOiJhbm5vbnltb3VzIiwiam9iVGl0bGUiOiIiLCJkb2IiOiIyMDE4LTA0LTE0VDA2OjI4OjEzLjgwM1oiLCJjcmVhdGVkT24iOjE1MjM2ODcyOTM4MDMsImlzVmFsaWRhdGVkIjp0cnVlLCJsYW5ndWFnZSI6IkVuZ2xpc2giLCJtb2JpbGVQaG9uZSI6IiIsImhvbWVQaG9uZSI6IiIsImJ1c2luZXNzUGhvbmUiOiIiLCJmYXgiOiIiLCJhZGRyZXNzIjp7ImFkZHJlc3NMaW5lMSI6IiIsImFkZHJlc3NMaW5lMiI6IiIsInN0YXRlIjoiIiwiY2l0eSI6IiIsImNvdW50cnkiOiIiLCJ6aXBDb2RlIjoiIn0sImRpc3BsYXlOYW1lIjoiIiwiaGFzQWNjZXB0ZWRUZXJtcyI6ZmFsc2UsIm1GQUF1dGhlbnRpY2F0ZWQiOmZhbHNlLCJtRkFFbmFibGVkIjpmYWxzZSwiX192IjowLCJwYXJ0eVNmZGNJZCI6bnVsbCwib3JnYW5pemF0aW9uTmFtZSI6bnVsbCwib3JnYW5pemF0aW9uSWQiOm51bGwsIm9yZ2FuaXphdGlvbiI6bnVsbCwibGFzdE5hbWUiOm51bGwsImZpcnN0TmFtZSI6bnVsbCwiaWQiOiIiLCJvcmdhbml6YXRpb25zIjpudWxsLCJ1c2VyVHlwZSI6ImFub255bW91cyIsImNvbnRhY3QiOnt9fSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy50YXlsb3JmcmFuY2lzLmNvbS9pZGVudGl0eS8iLCJleHAiOjE1MjM2OTA4OTQsImlhdCI6MTUyMzY4NzI5NCwiYXVkIjoiMWU0YTcxMjdkNzllODM3MjE0YmE2NDMxNTZlMzdmNTk5ZDBjMmNkMTVjNjlkMWIyZDMxY2RjZjllZTIyNzlkMCIsInN1YiI6IjVhMzBkMGU0NTc0Mjk3MWIwMDljOTIwZiJ9.yfSJw3OrdQDZMkA7KjsKUsNwgn3atphTgkUcjdW7HWo'


success_count = 0
fail_count = 0


# 写入大文件中
def write2bigfile(isbn, text):
    file_dict = dict()
    file_dict['isbn'] = isbn
    file_dict['content'] = text
    file_str = json.dumps(file_dict)
    dir_path = './DetailPage'
    if not os.path.exists(dir_path):
        os.mkdir(dir_path)
    file_path = './DetailPage/' + isbn[:-6] + '.txt'
    with open(file_path, 'a', encoding='utf-8') as f:
        f.write(file_str + '\n')
        f.close()
        logging.info('Writing into Big File*.txt: %s' % isbn)


# 写入日志
def write2log(log_text):
    dir_path = './detail_log'
    if not os.path.exists(dir_path):
        os.mkdir(dir_path)
    file_log_path = './detail_log/issn.log'
    with open(file_log_path, 'a') as f:
        f.write(log_text)


# 更新数据库
def updateDB(conn, issn, success=False):
    cursor = conn.cursor()
    if success:
        sql = '''UPDATE book SET stat = 1 where isbn13 = "%s"''' % issn
    else:
        sql = '''UPDATE book SET failcount = failcount + 1 WHERE isbn13 = "%s" ''' % issn
    cursor.execute(sql)
    conn.commit()


# 获取下载任务
def getTasksFromDB(conn):
    result_list = []
    curcor = conn.cursor()
    sql = "SELECT isbn13 FROM book where stat = 0 and failcount < 3  ORDER BY rand() limit 10"
    curcor.execute(sql)
    result_raw = curcor.fetchall()
    for r in result_raw:
        if r is not None:
            result_list.append(r[0])
    return result_list


# 请求API获取数据
def get2API(session, isbn, conn):
    status = False
    global success_count, fail_count
    api_url = 'https://api.taylorandfrancis.com/v2/products/titles/' + isbn
    resp = session.get(api_url)
    if resp.status_code == 200 or resp.status_code == 304:
        try:
            resp_dict = json.loads(resp.text)
            if resp_dict['metadata']['status'] == 'success':
                status = True
                write2bigfile(isbn, resp.text)
                logging.info('Success: %s' % isbn)
                success_count += 1
        except:
            logging.exception('Failed: %s Response is Error (is not a dict)' % isbn)
            fail_count += 1
    else:
        logging.exception('Failed: %s Response is Error (status_code is error)' % isbn)
        fail_count += 1
    # updateDB(conn, isbn, status)
    time.sleep(0.5)


def start():
    conn = pymysql.connect(host=DB_CONFIG['host'], user=DB_CONFIG['user'], passwd=DB_CONFIG['passwd'],
                           db=DB_CONFIG['db'], charset='utf8mb4', port=DB_CONFIG['port'])
    session = requests.Session()
    session.headers = COMMENT_HEADERS
    while True:
        session.get(url=TF_HOME_URL, headers=COMMENT_HEADERS)  # 请求主页获取cookie,headers信息
        id_token = session.cookies['books_token']
        # print(id_token)
        COMMENT_HEADERS['authorization'] = 'idtoken ' + id_token
        result = getTasksFromDB(conn)
        if not result:
            break
        for each_isbn in result:
            get2API(session, each_isbn, conn)
        logging.info('Success: %s, Failed: %s' % (success_count, fail_count))
    logging.info('All >> Success: %s, Failed: %s' % (success_count, fail_count))  # 完成爬取


if __name__ == '__main__':
    start()
